import numpy as np
import pandas as pd
import os
import tarfile
import urllib

import urllib3

import matplotlib.pyplot as plt

pd.set_option('display.width', 1000)  # 适应宽屏显示器
# 设置最大显示行数（默认 60）
pd.set_option('display.max_rows', 100)

# 设置最大显示列数（默认 20）
pd.set_option('display.max_columns', 100)

DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("datasets", "housing")
HOUSING_URL =DOWNLOAD_ROOT +"datasets/housing/housing.tgz"


def fetch_housing_data_lib3(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    # 创建目录（如果不存在）
    os.makedirs(housing_path, exist_ok=True)
    tgz_path = os.path.join(housing_path, "housing.tgz")

    # 使用urllib3下载文件
    http = urllib3.PoolManager()
    response = http.request('GET', housing_url)

    # 将下载的内容写入文件
    with open(tgz_path, 'wb') as f:
        f.write(response.data)

    # 解压文件
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    os.makedirs(housing_path, exist_ok=True)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    print(tgz_path)
    print(dir(urllib))
    urllib.request.urlretrieve(housing_url, tgz_path)

    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=HOUSING_PATH)
    housing_tgz.close()

def load_housing_data(housing_url=HOUSING_URL):
    csv_path = os.path.join(HOUSING_PATH, "housing.csv")
    return pd.read_csv(csv_path)

def test_housing_data():
    housing_data = load_housing_data()
    print(housing_data.head())
    print(housing_data.info())
    print("====================value counts=========================")
    print(housing_data['ocean_proximity'].value_counts())
    print("======================describe=====================================")
    print(housing_data.describe())

def test_data_hist():
    data = load_housing_data()
    data.hist(bins=50,figsize=(20,10))
    plt.show()


def split_train_test(data, test_ratio):
    # 设置随机数种子，保证相同的 data 数据集 每次产生的效果一样，但是 data 数据集改变了，就会产生不一样的结果
    np.random.seed(42)
    shuffled_indices = np.random.permutation(len(data))
    test_set_size = int(len(data) * test_ratio)
    test_indices = shuffled_indices[:test_set_size]
    train_indices = shuffled_indices[test_set_size:]
    return data.iloc[train_indices], data.iloc[test_indices]

from zlib import crc32

def test_set_check(identifier, test_ration):
    return crc32(np.int64(identifier)) & 0xffffffff < test_ration * 2 ** 32

def split_train_test_by_id(data,test_ration, id_column):
    """

    :param data:
    :param test_ration:
    :param id_column:
    :return:
    """
    ids = data[id_column]
    in_test_set =ids.apply(lambda id_: test_set_check(id_,test_ration))
    return data.loc[~in_test_set],data.loc[in_test_set]

def test_split_train_test_by_id():

    data = load_housing_data()
    # data 没有 id 列，需要我们手动的添加
    housing_with_id = data.reset_index() # 添加 index column
    # 构建更稳定的 id column , 经纬度的方式
    housing_with_id['id'] = data['longitude'] * 1000 + data['latitude']
    train_set, test_set = split_train_test_by_id(housing_with_id,0.2, "index")

    print(f" train_set head data is : \r\n {train_set.head()},test_set head data is :\r\n {test_set.head()}")

def test_split_train_test():
    data = load_housing_data()
    train_set, test_set = split_train_test(data,0.2)
    print(f"{len(train_set)} train samples")
    print(f"{len(test_set)} test samples")

from sklearn.model_selection import train_test_split

def split_data_by_sklearn(data,radio):
    train_set, test_set = train_test_split(data,test_size=radio,random_state=42)
    print(f" train_set head data is : \r\n {train_set.head()},test_set head data is :\r\n {test_set.head()}")
    return train_set,test_set

def income_cat_cut():
    data = load_housing_data()
    data['income_cat'] = pd.cut(data['median_income'],bins=[0.,1.5,3.0,4.5,6.0,np.inf], labels=[1,2,3,4,5])
    print(data.head(20))

    data['income_cat'].hist()
    plt.show()

from sklearn.model_selection import StratifiedShuffleSplit

def split_by_income_cat():
    data = load_housing_data()
    data['income_cat'] = pd.cut(data['median_income'],bins=[0.0,1.5,3.0,4.5,6.0,np.inf],labels=[1,2,3,4,5])

    split = StratifiedShuffleSplit(n_splits=1,test_size=0.2,random_state=42)
    for train_index,test_index in split.split(data,data['income_cat']):
        strat_train_set = data.loc[train_index]
        start_test_set = data.loc[test_index]
        print(f"level split radio : {start_test_set['income_cat'].value_counts()/len(start_test_set)}")

        # 恢复数据
        for set_ in (start_test_set,strat_train_set):
            set_.drop("income_cat",axis = 1,inplace= True)

def test_data():
    data = load_housing_data()
    train_set,test_set = split_data_by_sklearn(data,0.2)
    housing = train_set.copy()

    housing.plot(kind='scatter',x='longitude',y='latitude',
                 alpha=0.4,s=housing['population']/100,label='population',figsize=(10,7),c="median_house_value",cmap=plt.get_cmap('jet'),colorbar=True,)
    housing = housing.drop('ocean_proximity',axis=1)
    plt.show()

    print("coor data is : \r\n")
    corr_matrix = housing.corr()
    result = corr_matrix['median_house_value'].sort_values(ascending=False)
    print(result)
    # plt.legend()

from pandas.plotting import scatter_matrix

def test_scatter_matrix():
    data = load_housing_data()
    attributes = ['median_house_value',"median_income","total_rooms","housing_median_age"]

    scatter_matrix(data[attributes],figsize=(12,8))
    data.plot(kind='scatter',x="median_income",y="median_house_value",alpha=0.1)
    plt.show()

def test_data_relation():
    data = load_housing_data()
    data["rooms_per_household"] = data["total_rooms"]/data["households"]
    data['bedrooms_per_room'] = data["total_bedrooms"]/data["total_rooms"]
    data['population_per_household'] = data["population"]/data["households"]

    data = data.drop('ocean_proximity',axis=1)
    corr_matrix = data.corr()
    result = corr_matrix["median_house_value"].sort_values(ascending=False)
    print(result)

def split_feature_label():
    data = load_housing_data()
    data = data.drop("median_house_value",axis=1)
    data_labels = data["median_house_value"]


def test_miss_data():
    """
    数据缺失处理：
        1. 放弃这些数据
        2. 放弃整个属性
        3. 缺失值的填充(0,中位数，平均数)
    :return:
    """
    data = load_housing_data()
    # 放弃空数据行
    data1 = data.dropna(subset = ['total_bedrooms'])
    print(f"放弃空行数据后的结果:{data1.info()}")
    # 直接放弃整个属性
    data2 = data.drop("total_bedrooms",axis=1)
    print(f"放弃整个属性后的结果是{data2.info()}")
    # 用中位数填充
    median = data['total_bedrooms'].median()
    data3 = data['total_bedrooms'].fillna(median)
    print(f"用中位数填充后的结果是:{data3.info()}")

from sklearn.impute  import SimpleImputer
def test_missing_data_by_sklearn():
    data = load_housing_data()
    imputer = SimpleImputer(strategy='median')
    housing_num = data.drop('ocean_proximity',axis=1)
    imputer.fit(housing_num)
    print(imputer.statistics_)
    print(housing_num.median().values)
    print(housing_num.info())

    result = imputer.transform(housing_num)





if __name__ == "__main__":
    # 获取数据
    # fetch_housing_data(HOUSING_URL)
    # fetch_housing_data_lib3(HOUSING_URL, HOUSING_PATH)
    # 加载数据
    # test_housing_data()
    # test_data_hist()
    # test_split_train_test()
    # test_split_train_test_by_id()
    # split_data_by_sklearn(load_housing_data(),0.2)
    # income_cat_cut()
    # split_by_income_cat()
    # test_data()
    # test_scatter_matrix()
    # test_data_relation()
    # test_miss_data()
    test_missing_data_by_sklearn()